# -*- coding:utf-8 -*-

import os
import sys
import fnmatch
import time
import threading
import thread
import shutil
import copy

from MyException import Error, MatchRuleError, GetSizeError, ReadFileError, MoveFileError, DelFileError,MemoryOverFlowError
from MyLogger import MyLogger, SIZEROTATE, TIMEROTATE
from StatInfo import StatInfo
from StatLog import StatLog
from Match import MatchResult
     
class ReSendMatch(threading.Thread):
    def __init__(self, profile, match_rule_list, nomatch_rule_list, statlog_map, send_file_list,fsinfo):
        """
        初始化
        """
        threading.Thread.__init__(self)
        
        self.profile = profile
        
        self.lock = threading.RLock()
        
        self.match_rule_list = match_rule_list
        self.nomatch_rule_list = nomatch_rule_list
        
        self.statlog_map = statlog_map
                
        self.send_file_list = send_file_list
        
        self.fsinfo = fsinfo
        
        self.logger = MyLogger(os.path.join(profile.log_dir, "ReSendMatch_Result.log"), SIZEROTATE, profile.size_rotate_backup_num)
        
        self.statlog = None

            
    def create_statlog(self, file_path):
        """
        创建统计日志信息
        """
        statinfo = StatInfo()
        file_name = os.path.basename(file_path)
        
        #获取文件大小
        try:
            statinfo.file_size = str(os.path.getsize(file_path))
        except os.error, msg:
            raise GetSizeError(msg)
        
        if self.profile.is_write_statlog:   #判断是否需要写统计日志
            #获取文件类型
            index = file_name.rfind(".")
            if index > -1:
                file_type = file_name[index+1:].lower()
            else:
                file_type = "--"
            statinfo.file_type = file_type
            
            #获取地市编号，协议类型，数据源
            if file_type == "xml":
                list = file_name[:-4].split("_")
                if len(list) == 6:
                    statinfo.data_source = list[5]
                    statinfo.protype_desc = list[3]
                    statinfo.uparea_id = list[4]
                    statinfo.file_time = list[1]
            elif file_type == "bcp":
                list = file_name[:-4].split("_")
                if len(list) == 6:
                    statinfo.data_source = list[5]
                    statinfo.protype_desc = list[3]
                    statinfo.uparea_id = list[4]
                    statinfo.file_time = list[1]
                elif len(list) == 8:
                    statinfo.data_source = list[7]
                    statinfo.protype_desc = list[4]
                    statinfo.uparea_id = list[6]
                    statinfo.file_time = list[1]
            elif file_type == "zip":
                list = file_name[:-4].split("_")
                if len(list) == 5:
                    statinfo.uparea_id = list[1]
                    statinfo.data_source = list[2]
                    statinfo.file_time = list[3]
            else:
                pass
            
            #获取数据日期
            statinfo.data_date = time.strftime("%Y%m%d", time.localtime())
            
            #获取记录数
            try:
                if file_type == "bcp":
                    fp = open(file_path)
                    lines = fp.readlines()
                    statinfo.record_num = str(len(lines))
            except Exception, msg:
                raise ReadFileError(msg)
        
        return statinfo
        
    '''
    匹配备份目录下面的文件
    ftpOrPath_GolbalName：ftp或者path的全局名称
    匹配不上时候，返回:None,否则返回：MatchResult
    add by wshi 2012-09-21
    '''    
    def match_backfile(self, ftpOrPath_GolbalName):
        #初始化匹配规则
        match_result = MatchResult(None)
        
        isFind = False
        
        if ftpOrPath_GolbalName.find("_FTP_") > 0:
            #匹配所有一般规则
            for match_rule in self.match_rule_list:
                #查找ftp接口
                for ftparry in match_rule.ftp_list:
                    if ftparry[0] == ftpOrPath_GolbalName:
                        if ftparry[1]:
                            match_result.ftp_list = [ftparry]
                            isFind = True
                            break
                if isFind:
                    break
        elif ftpOrPath_GolbalName.find("_PATH_") > 0:
            #匹配所有一般规则
            for match_rule in self.match_rule_list:
                #查找ftp接口
                for patharry in match_rule.path_list:
                    if patharry[0] == ftpOrPath_GolbalName:
                        if patharry[1]:
                            match_result.path_list = [patharry]
                            isFind = True
                            break
                if isFind:
                    break
                
        if isFind:
            return match_result
        else:
            return None
        
        
    """
    获取一个最新可用的日期目录
    sub_backup_dir：备份文件的子目录全路径
    date_subdir_list：备份目录下面日期子目录
    没有返回None，否则返回日期目录名称
    add by wshi 2012-09-21
    """
    def getnewfilelst(self,sub_backup_dir,date_subdir_list):
        #
        dateDirNum = len(date_subdir_list)
        for i in range(dateDirNum):
            #查找最新的备份文件
            max_time_dir = max(date_subdir_list)
            
            #判断最新目录是否在有效期内,如果不在，直接返回None，说明没有可用的文件
            dir_time = time.mktime(time.strptime(max_time_dir, "%Y%m%d%H"))
            if (time.time() - dir_time) > (self.profile.backup_file_valid_time):
                #self.logger.info("unvalid dir:%s by %d > %d" %(max_time_dir,time.time() - dir_time,self.profile.backup_file_valid_time))
                return None
            
            #空目录，继续取下个目录
            filepath = os.path.join(sub_backup_dir,max_time_dir)
            if os.path.exists(filepath):
                fileLst = os.listdir(filepath)
                if len(fileLst) < 1:
                    #self.logger.info("empty dir:%s,len:%s" %(filepath,str(len(fileLst))))
                    date_subdir_list.remove(max_time_dir)
                    continue
                else:
                    return max_time_dir
        return None
    
    """
    获取ftp接口状态或者path的状态
    match_result：匹配的结果
    返回：True：接口可用，False：接口不可用
    add by wshi 2012-09-21
    """
    def get_resendstatus(self,match_result):
        if len(match_result.ftp_list) > 0:
            #ftp接口全部不管用
            #if match_result.ftp_list[0][1].invalid_ftp_list.size() < len(match_result.ftp_list[0][1].ftp_put_list):
            if match_result.ftp_list[0][1].valid_ftp_list.size() > 0:
                return True
            else:
                return False
        elif len(match_result.path_list) > 0:
            if os.path.exists(match_result.path_list[0][1].tmp_dir) and os.path.exists(match_result.path_list[0][1].dst_dir):
                return True
            else:
                return False
        return False
           
    """
    将要发送目录下面的文件添加到待发送队列里面
    dir_path：发送文件目录
    usable_move_size：运行发送总大小
    match_result:文件匹配对象
    返回：剩余可以移动的内存使用率
    add by wshi 2012-09-21
    """ 
    def add_resendfile(self,dir_path,usable_move_size,par_match_result):
        file_list = os.listdir(dir_path)
                    
        #逐个移动缓存文件，最大只移动内存盘大小10%
        for filename in file_list:
            try:
                #获取内存盘使用率
                self.fsinfo.update_mem_info()
                mem_used = self.fsinfo.mem_used
                #内存使用率大于最大使用率
                if mem_used > self.profile.mem_free_used:
                    raise MemoryOverFlowError("stop resendmatch,the memory has used:%d bigger than the %d" %(mem_used,self.profile.mem_free_used))
                
                #回填具体的文件名
                match_result = MatchResult(filename)
                match_result.ftp_list = []
                match_result.ftp_list.extend(par_match_result.ftp_list)
                match_result.path_list = []
                match_result.path_list.extend(par_match_result.path_list)
                
                src_file = os.path.join(dir_path, filename)
                
                #生成统计日志信息
                statinfo = self.create_statlog(src_file)
            
                #获取文件大小
                try:
                    filesize = os.path.getsize(src_file)
                except Exception, msg:
                    raise GetSizeError(msg)
                
                #移动文件到内存盘
                try:
                    dst_file = os.path.join(self.profile.send_dir, filename)
                    shutil.move(src_file, dst_file)                    
                except Exception, msg:
                    raise MoveFileError(msg)
                
                #将统计日志信息和匹配规则加入队列
                self.statlog_map.insert(filename, statinfo)
                self.send_file_list.add(match_result)
                self.logger.info("Success resendmatch file: %s size: %s record: %s matches: 1"%(filename, statinfo.file_size, statinfo.record_num))
                
                #移动总量超过内存盘大小则跳出循环
                usable_move_size -= filesize
                if usable_move_size < 0:
                    raise MemoryOverFlowError("stop resendmatch,this times has moved past memory's ten percent ,now memory:%d"%mem_used)
                
            except MemoryOverFlowError,msg:
                self.logger.error("Error move back file: %s cause: MoveFile Error:%s"%(filename,str(msg)))
                raise
            except GetSizeError, msg:
                self.logger.exception("Error resendmatch file: %s size: 0 record: 0 cause: GetSize Error:%s"%(filename,str(msg)))
            except ReadFileError, msg:
                self.logger.exception("Error resendmatch file: %s size: 0 record: 0 cause: ReadFile Error:%s"%(filename,str(msg)))
            except MoveFileError, msg:
                self.logger.exception("Error resendmatch file: %s size: %s record: %s cause: MoveFile Error:%s"%(filename, statinfo.file_size, statinfo.record_num,str(msg)))
                raise                          
            except Exception, msg:
                self.logger.exception("Error resendmatch file: %s szie: %s record: %s cause:  Error:%s"%(filename, statinfo.file_size, statinfo.record_num,str(msg)))
        #删除目录
        if usable_move_size >= 0:
            try: 
                shutil.rmtree(dir_path)
            except: 
                pass
        return usable_move_size
                        
    def run(self):
        """
        匹配线程执行主体
        """
        while(True):
            try:
                #获取内存盘大小，并计算内存盘大小的10%
                self.fsinfo.update_mem_info()
                mem_size = self.fsinfo.mem_size
                usable_move_size = int(mem_size*1024*1024*0.1)
                self.logger.info("start memory ten percenet size:%d" %usable_move_size)
                    
                #扫描备份目录，获取文件名
                dir_list = os.listdir(self.profile.file_backup_dir)
                for ftporpath_globalname in dir_list:
                    try:
                        #确定是可用的重发接口
                        match_result = self.match_backfile(ftporpath_globalname)
                        if not match_result:
                            self.logger.info("not find the gloabname:%s" %ftporpath_globalname)
                            continue
                        
                        #目录下是否有时间子目录
                        sub_backup_dir = os.path.join(self.profile.file_backup_dir, ftporpath_globalname)
                        date_subdir_list = os.listdir(sub_backup_dir)
                        if not date_subdir_list:
                            self.logger.info("the dir:%s is empty" %sub_backup_dir)
                            continue
                        
                        #获取一个可以上传的最新的文件夹
                        max_time_dir = self.getnewfilelst(sub_backup_dir,date_subdir_list)
                        if not max_time_dir:
                            self.logger.info("the dir:%s has not newer date dir" %sub_backup_dir)
                            continue;
                        
                        #获取内存盘使用率
                        self.fsinfo.update_mem_info()
                        mem_used = self.fsinfo.mem_used
                        #内存使用率大于最大使用率
                        if mem_used > self.profile.mem_free_used:
                            raise MemoryOverFlowError("stop resendmatch,the memory has used:%d bigger than the %d" %(mem_used,self.profile.mem_free_used))
                                
                        #判断接口是否可用，当前重发接口不可用，循环下一个重复目录
                        if not self.get_resendstatus(match_result):
                            self.logger.info("the %s has not any used interface" %ftporpath_globalname)
                            continue
                        
                        #扫描最近时间目录下的文件
                        dir_path = os.path.join(sub_backup_dir, max_time_dir)
                        
                        usable_move_size = self.add_resendfile(dir_path,usable_move_size,match_result)
                        self.logger.info("memory ten percenet size leaving :%d" %usable_move_size)   
                    except MemoryOverFlowError,msg:
                        raise
                    except Exception, msg:
                        self.logger.exception("resend the %s error:%s" %(ftporpath_globalname,str(msg)))
            
            except Exception, msg:
                self.logger.exception("error:%s" %str(msg))    
            except:
                self.logger.exception("Unknown error")
            
            #5分钟重发一次备份文件
            time.sleep(300)
        
import unittest
import ConfigInfo
import ConfigReader
import ConfigAssist
import MutexMap
import MutexList

class MatchTestCase(unittest.TestCase):
    def setUp(self):
        pass
    
    def tearDown(self):
        pass
            
    def testBaseFunc(self):
        profile = ConfigReader.ConfigReader().read_profile()
        profile.is_write_statlog = True
        all_app_conf, result = ConfigAssist.ConfigAssist().check_match_rule()
        all_app_conf.show()
        
        statinfo_map = MutexMap.MutexMap()
        send_file_list = MutexList.MutexList()
        
        filename = "MASS_1234567890_123456_HTTP_320100_111.bcp"
        fp = open(os.path.join(profile.work_dir, filename), "w")
        fp.write("dsfasadf")
        fp.close()
        
        match = Match(profile, all_app_conf.match_rule_list, all_app_conf.nomatch_rule_list, statinfo_map, send_file_list)
        match.daemon = True
        match.start()
        
        time.sleep(3)
        
        statinfo = statinfo_map.find(filename)
        statinfo.show()
        print send_file_list.copy()[0].show()
        
if __name__ == "__main__":
    suite = unittest.TestSuite()
    suite.addTest(MatchTestCase("testBaseFunc"))  
        
    runner = unittest.TextTestRunner()
    runner.run(suite)

